package com.hailiang.reptile;


import com.hailiang.scheduler.UrlSchedulerImp;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;

import java.io.IOException;
import java.util.HashMap;

public class Reptile {
    private HtmlCleaner htmlCleaner = new HtmlCleaner();
    private HttpClient httpClient = HttpClients.createDefault();
    private HashMap nbMap = JedisUtil.getMap();
    private UrlSchedulerImp urlSchedulerImp = new UrlSchedulerImp();
   private static Reptile rep = new Reptile();
    //总方法
    public void reptile(String url) throws Exception {
        Integer nb = (Integer) nbMap.get("nb");
        String newUrl = url+nb;
        nb+=50;
        nbMap.put("nb",nb);
        String html = downLoad(newUrl);
        Object[] objects = parse(html);
        for (Object object : objects) {
            TagNode tagNode = (TagNode)object;
            String href = tagNode.getAttributeByName("href");
            String content = tagNode.getText().toString();
            urlSchedulerImp.push("url",href);
        }
        rep.reptile("https://tieba.baidu.com/f?kw=%E7%BE%8E%E5%A5%B3&ie=utf-8&pn=");
    };

    /**
     *解析
     */
    public Object[] parse(String html) throws Exception {
        TagNode tagNode = htmlCleaner.clean(html);
        Object[] objects = tagNode.evaluateXPath("//a[@class='j_th_tit ']");
        System.out.println(objects.length);
        return objects;
    }
        //下载网页
    public String downLoad(String url) throws IOException {
        HttpGet httpGet = new HttpGet(url);
        HttpResponse httpResponse = httpClient.execute(httpGet);
        String s = EntityUtils.toString(httpResponse.getEntity());
        return s;
    }
}
